if __name__ == '__main__':
    pass

from html.parser import HTMLParser
from datetime import datetime
import re
import pymongo
from fill_mongo import Games
from common import Download_Games
import os
 
class LFCHistoryParser_2014(HTMLParser):
    
    def __init__(self, dict):
        self.strict = True
        self.dict = dict
        self.mode = ""
        self.last_data = ""
        self.line = []
        self.record = []
        
        self.stat_tags = ['Game date', 'Stadium', 'Competition', 'Attendance', 'Half-time score',\
                         'Manager', 'Opposition manager', 'Referee']
        
        self.main_tags = {  'Starting lineup':(3, 4),\
                            'Subs': (3, 4),\
                            'Goals': (1, 2, 3),\
                            'Red Cards':(2,3),\
                            'Sent off': (2,3),\
                            'Substitutions': (5, 6),\
                            'Opposition lineup': (18, 22), 
                            'Opposition substitutions': (3, 5, 6),\
                            'Opposition goals': (1, 2, 3),\
                            'Notes': range(0,10)}
    
    def handle_starttag(self, tag, attrs):
        managers_path = '/Managers/Manager/Profile/'
        
        if self.mode=='':
            ''' no active mode '''
            if tag == 'table' and attrs[0][1] == 'game-stats':
                self.mode='game-stats'
       
        elif (not self.mode == '') and (tag == 'h3'):
            ''' Emtpy section, close it '''
            self.line = []
            self.mode = ''

        if tag == 'item':
            ''' Specify which links to parse inside each section '''
            if self.mode == 'game-stats':
                tags = ('/Managers/Manager/Profile', '/Opposition/Manager/Profile', '/Referees/Referee/Profile')
            elif self.mode in ('Opposition lineup', 'Opposition substitutions', 'Opposition goals'):
                tags = ('/Opposition/Players/Profile')
            elif self.mode in ('Starting lineup', 'Subs', 'Goals', 'Substitutions', 'Red Cards', 'Sent off') :
                tags =  ('/Players/Player/Profile')
            else:
                tags = ()

            ''' Split the link '''
            if attrs[0][1].count('/') == 4 and attrs[0][1].rpartition('/')[0] in tags:
                self.line.append(attrs[0][1].split('/')[-1])

                 
    def handle_endtag(self, tag):
        if (tag == 'div') or (tag == 'table' and self.mode == 'game-stats'):
            ''' Stop parsing if empty section or initial stats'''
            self.line = []
        
        elif self.mode in self.main_tags.keys():
            
            if tag == 'tr':
                ''' Verify data and append to the list '''
                assert len(self.line) in self.main_tags[self.mode], 'Data issue:' + str(self.line)
                self.record.append(self.line)
                self.line = []
            
            elif tag == 'table':
                ''' save list and clear mode '''
                self.dict[self.mode]=self.record
                self.record = []
                self.mode = ''      
            
    
    def handle_data(self, data):
        if (len(data.rstrip()) >  0) and (not data[0]==','):
 
            if self.mode == '':
                ''' start most modes'''
                if data in self.main_tags.keys():
                    self.mode = data
             
            elif self.mode in self.main_tags.keys():
                ''' capture data '''
                self.line.append(data)

            elif self.mode == 'game-stats':
                ''' specifically process general stats'''
                if self.last_data in self.stat_tags:
                    if self.line:
                        self.dict[self.last_data] = [self.line[0], data]
                        self.line.pop(0)
                    else:
                        self.dict[self.last_data] = data
                    
                self.last_data = data.replace(':', '')
                

class LiverwebParser_2014(HTMLParser):
    def __init__(self, dict):
        self.strict = True
        self.dict = dict
        self.mode = ''
        self.tag = ''
        self.cache = []
        self.stats_table = []
        self.exception_data=['Follow on Twitter :','@Liverweb']
        self.opposition = ''
        self.liverpool_tags = ('Liverpool', 'Unused Subs', 'Goal(s)', 'Bookings', 'Red Cards', 'Shirt Worn', 'Manager')
        self.opposition_tags =            ( 'Unused Subs', 'Goal(s)', 'Booking',  'Red Cards', 'Shirt Worn', 'Manager')
        self.match_tags = ('Referee', 'Att', 'Venue', 'Kick Off', 'Date', 'HT/FT', 'Competition')

 
    def next_mode(self, mode):
        if self.mode:
            self.dict[self.mode] = self.cache
            self.cache = []
            self.tag = ''
        
        self.mode = mode
        #print ('--Mode: ', self.mode)        
 
    def handle_starttag(self, tag, attrs):
        #print ('>', tag, ':' , attrs)
        
        if self.mode == '' and tag == 'font':
            self.next_mode('Score')
        
        elif self.mode == 'Report' and tag == 'hr':
            self.next_mode('Liverpool data')
            
        elif self.mode == 'Liverpool data' and tag == 'hr':
            self.next_mode('Opposition data')

        elif self.mode == 'Opposition data' and tag == 'hr':
            self.next_mode('Match data')

        elif self.mode == 'Match data' and tag == 'hr':
            self.next_mode('Stats')

        elif self.mode == 'Stats' and tag == 'table':
                self.mode = 'Stats table'

  
    def handle_endtag(self, tag):
        #print ('<', tag)
        
        if self.mode == 'Score' and tag == 'font':
            self.next_mode('Report')
            
        elif self.mode == 'Stats table' and tag == 'tr':
            self.cache.append(self.stats_table)
            self.stats_table = []

        elif self.mode == 'Stats table' and tag == 'table':
            self.next_mode('Notes')
                        
    def handle_data(self, data):
        d = data.strip()
        if (len(d) >  0) and (not d in self.exception_data):
            #print (d)        
            
            if self.mode == 'Liverpool data':
                if data in self.liverpool_tags:
                    self.tag = data
                elif self.tag:
                    self.cache.append({self.tag:data})
                    self.tag =''
                else:
                    assert False, data
                
            elif self.mode == 'Opposition data':
                ''' assert first data is opposition name '''
                if not self.opposition:
                    self.opposition = data
                if data in self.opposition_tags + (self.opposition, ):
                    self.tag = data
                elif self.tag:
                    self.cache.append({self.tag:data})
                    self.tag =''
                else:
                    assert False, data

            elif self.mode == 'Match data':
                if data in self.match_tags:
                    self.tag = data
                elif self.tag:
                    self.cache.append({self.tag:data})
                    self.tag =''
                elif (data.find('Match shown') > 0):
                    assert False, data

            elif self.mode == 'Stats table':
                self.stats_table.append(data)

            elif self.mode == 'Notes':
                #print (data)
                if data in ('Match Programme', 'League Stats at end of match day'):
                    self.next_mode('')
                else:
                    self.cache.append(data)

            elif self.mode:
                self.cache.append(d)
        
        
            
class Parse_LiverWeb():
    def __init__(self, remote = False):
        self.base_url = 'http://www.liverweb.org.uk/report.asp?rec_id='
        self.start_tag = "Liverweb Match Report "
        self.end_tag = "Match Programme"
        self.code_page = 'cp1252'
 
        if remote:
            self.connection = pymongo.MongoClient("localhost", 27017).test
            self.game_db = Games(self.connection.games_w)
        else:
            self.game_db = False
 
        
    def download_html(self, path, range):
        loader = Download_Games(self.base_url, range)
        loader.create_files(path, self.code_page)

    def write_mongo(self, line):
        if self.game_db:
            self.game_db.write_stage_line(line)

    def list2dict(self, list):
        dict = {}
        for item in list:
            for key in item:
                dict[key] = item[key]
        return dict
            
    def read_file(self, path, id):
        infile = '{}/game_{}.txt'.format(path, id) 

        game_data = {}
        parser = LiverwebParser_2014(game_data)
        parser.reset()
        
        game_data["wid"] = id
            
                    
        data = open(infile, 'r', encoding = self.code_page).read()
        start = data.find(self.start_tag) + len(self.start_tag)
        end = data[start:].find(self.end_tag) + len(self.end_tag)+ start 
        assert start > 0 and end > start, 'Tags not found'
        parser.feed(data[start:end].replace(' & ', ' and '))
            
        match_data_dict = self.list2dict(game_data['Match data'])
        game_data.pop('Match data')
        game_data.update(match_data_dict)
        
        print (game_data)


    def read_files(self, path):
        for infile in os.listdir(path):
            game_data = {}
            parser = LiverwebParser_2014(game_data)
            parser.reset()
            
            print ("current file is:", path+infile)
            
            game_data["wid"] = infile[infile.find('_')+1:infile.find('.')]
            
                    
            data = open(path+infile, 'r', encoding = self.code_page).read()
            start = data.find(self.start_tag) + len(self.start_tag)
            end = data[start:].find(self.end_tag) + len(self.end_tag)+ start 
            assert start > 0 and end > start, 'Tags not found'
            parser.feed(data[start:end].replace(' & ', ' and '))
            
            match_data_dict = self.list2dict(game_data['Match data'])
            game_data.pop('Match data')
            game_data.update(match_data_dict)

            self.write_mongo(game_data)
            
            #assert len(game_data.keys()) in range(7, 20), 'Keys missing: ' + game_data['id'] + str(game_data.keys())
            #print (game_data)
            #for key in game_data.keys():
            #    if not key == 'Report':
            #        print (key, ' : ', game_data[key])
            #break


class Parse_Html_Games():    
    def __init__ (self, remote = False):
        if remote:
            self.connection = pymongo.MongoClient("localhost", 27017).test
            self.game_db = Games(self.connection.games)
        else:
            self.game_db = False
        
        self.start_tag = "<h2 class=\"page-name\">Report</h2>"
        self.end_tag = "<h3>Other games during this season</h3>"

         
        
    def write_mongo(self, line):
        if self.game_db:
            self.game_db.write_stage_line(line)
    
    def read_file(self, id, path = 'games_html'):
            game_data = {}
            parser = LFCHistoryParser_2014(game_data)
            parser.reset()
            
            infile = '{}/game_{}.txt'.format(path, id) 
                    
            data = open(infile, 'r', encoding = 'utf-8').read()
            start = data.find(self.start_tag) + len(self.start_tag)
            end = data[start:].find(self.end_tag) + start
            assert start > 0 and end > start, 'Tags not found'
            
            score_start = data[start:end].find('<h3>')+len('<h3>')
            score_end = data[start:end].find('</h3>')
            assert score_start and score_end and score_start < score_end
            game_data["Score"] = data[start+score_start:start+score_end]
            
            
            parser.feed(data[start:end])
            
            #print(game_data)
            self.write_mongo(game_data)

    
            
    def read_files(self, path):
        import os
        i = 0
        start_tag = "<h2 class=\"page-name\">Report</h2>"
        end_tag = "<h3>Other games during this season</h3>"
        
        for infile in os.listdir(path):
            game_data = {}
            parser = LFCHistoryParser_2014(game_data)
            parser.reset()
            
            #print ("current file is:", path+infile)
            
            game_data["hid"] = infile[infile.find('_')+1:infile.find('.')]
            
                    
            data = open(path+infile, 'r', encoding = 'utf-8').read()
            start = data.find(self.start_tag) + len(self.start_tag)
            end = data[start:].find(self.end_tag) + start
            assert start > 0 and end > start, 'Tags not found'
            
            score_start = data[start:end].find('<h3>')+len('<h3>')
            score_end = data[start:end].find('</h3>')
            assert score_start and score_end and score_start < score_end
            game_data["Score"] = data[start+score_start:start+score_end]
            
            
            parser.feed(data[start:end])
            assert len(game_data.keys()) in range(7, 20), 'Keys missing: ' + game_data['id'] + str(game_data.keys())
            
            self.write_mongo(game_data)
            
            
            '''
            i = i + 1
            if i==5000:
                print ('Cycle limit reached')
                break       
            '''











        
    def check_goalscorers(self, all_dicts):
        marksmen = {}
        for item in all_dicts:
            for [scorer, type, minute] in item['goals']:
                    
                if type == goal_type.own_goal:
                    continue
                    
                if scorer in marksmen:
                    marksmen[scorer] += 1
                else:
                    marksmen[scorer] = 1

        
        of_stat = []

        for line in open('data/temp', 'r', encoding='utf-8').readlines():
            items = line.split('\t')
            of_stat.append([items[0], items[1], items[2].strip()]) 


        count = 0
        for key, scored in zip(sorted(marksmen, key=marksmen.get, reverse=True), sorted(marksmen.values(), reverse=True)):
            for name in self.players.keys():
                if self.players[name] == key:
                    player = name
                    break
            else:
                assert False, key
    
            print (name, scored)
                
            if not int(of_stat[count][2]) == scored: 
                print (count, name, scored)
                print (of_stat[count], '\n')
    
            count += 1              
    
    def parse_void(self, text):
        print ('pv:', text)

    
    def get_section(self, text):
        functions = {' Subs '          : self.parse_sub,\
                    ' Goals '          : self.parse_goal,\
                    ' Substitutions '  : self.parse_substitutions,\
                    ' Red Cards '      : self.parse_red_cards,\
                    ' Notes '          : self.parse_notes,\
                    ' Opposition lineup ': 'exit',\
                    ' \n'              : 'exit'}
        
        try:
            return functions[text]
        except KeyError:
            return 'continue'
        
    
    def read_tournaments(self):
        self.tourn_dict = { 'European Cup Winners':    4653,\
                            '1st Division':            4654,\
                            '2nd Division':            4655,\
                            'Centenary Trophy':        5195,\
                            'Champions League':        310,\
                            'Champions L.':        310,\
                            'CL ':                  310,\
                            'Charity Shield':         4672,\
                            'Community Shield':        4673,\
                            'Europa League':        3243,\
                            'European Cup':            4675,\
                            'European Fairs Cup':    4676,\
                            'European Super Cup':    4677,\
                            'FA Cup'            :    154,\
                            'Lancashire League':    4678,\
                            'League Cup':            319,\
                            'Premier League':        69,\
                            'Screen Sport':            4679,\
                            'Sheriff of London':    4674,\
                            'Test Match':            4680,\
                            'UEFA Cup':                4682,\
                            'World Club':            4681
                         }
                



    def parse_tournament(self, line):
        for key in self.tourn_dict:
            if key in line:
                self.dict['tournament'] = self.tourn_dict[key]
                break
            elif line.strip().find('Eur. Cup') == 0:
                 self.dict['tournament'] = self.tourn_dict['European Cup']
                 break
            elif line.strip().find('SSSC') == 0:
                 self.dict['tournament'] = self.tourn_dict['Screen Sport']
                 break
            elif line.strip().find('Centenary') == 0:
                 self.dict['tournament'] = self.tourn_dict['Centenary Trophy']
                 break
            elif line.strip().find('ECW Cup') == 0:
                 self.dict['tournament'] = self.tourn_dict['European Cup Winners']
                 break
            elif line.strip().find('EFC ') == 0:
                 self.dict['tournament'] = self.tourn_dict['European Fairs Cup']
                 break
            elif line.strip().find('WCC ') == 0:
                 self.dict['tournament'] = self.tourn_dict['World Club']
                 break

        else:
            print ('Not found tournament', line)
            assert False, line


       
    def parse_referee(self, line):
        try:
            self.dict['referee'] = self.referees[line.strip()]
        except KeyError:
            name    = line.strip().split(' ')[0]
            initial = line.strip().split(' ')[1]
            for key in self.referees.keys():
                if key == 'Unknown':
                    continue

                k_name      = key.split(' ')[1]
                k_initial   = key.split(' ')[0][0]
                
                if name ==  k_name and initial  == k_initial:
                    #print ('__Assume {0} is {1}'.format(line, key))
                    self.dict['referee'] = self.referees[key]
                    break

                       
    
    def parse_line_up(self, line):
        record = []
        for i in range (0, 11):
            record.append([i+1, line[i*3].strip(), self.parse_player(line[i*3+1].strip() + ' ' + line[i*3+2].strip()), 0, 'NULL'])
        
        self.dict['line-up'] = record

    def parse_sub(self, line):
        self.dict['line-up'].append([len(self.dict['line-up'])+1, line[0].strip(), self.parse_player(line[1].strip() + ' ' + line[2].strip()), 'NULL', 'NULL'])
        return 3
        

    
    def parse_goal(self, line):
        shift = 1
        
        if line[0].strip() in ['\n', 'Substitutions', 'Red Cards']:
            ''' no goals '''
            return 0
                
        if line[0].find('O/G') > 0:
            ''' own goal '''
            scorer =  self.own_scorers[line[0].split('O/G')[0].strip()]
            minute =  line[0].split('O/G')[1].strip()[:-1]
            type = goal_type.own_goal
            if not minute.isdigit():
                minute = 0
                
            self.dict['goals'].append([scorer, type, minute])
            return shift
        else:
            scorer = self.parse_player(line[0].strip())
            if line[1].find(' pen ') > 0:
                type = goal_type.penalty
                minute = line[1].strip().split(' ')[0][:-1]
                shift = 2
            else:
                type = goal_type.ordinary
                minute = line[1].strip()[:-1]

            if minute.isdigit():
                shift = 2
            else:
                ''' no timestamp '''
                minute = 0
                
            self.dict['goals'].append([scorer, type, minute])
            return shift
               


    def parse_substitutions(self, line):
        coming_off = line[::4]
        coming_on =  line[2::4]
        time = line[3::4]
        
        assert (line.index(' Opposition lineup ') > 0)
        num_subs = int(line.index(' Opposition lineup ')/4)
        
        for i in range(0, num_subs):
            player_in = self.parse_player(coming_on[i])
            player_out = self.parse_player(coming_off[i])
            
            minute = time[i][2:-2]
            if not minute.isdigit():
                assert minute == ''
                minute = 0
                
            for item in self.dict['line-up']:
                if item[2] == player_in:
                    ''' set came off '''
                    item[3] = minute
                if item[2] == player_out:
                    ''' set came on '''
                    item[4] = minute
               
        ''' skipping space and minute sign '''
        return num_subs*4

    def generate_goals_stats_sql(self, out_file = ''):
        
        if len(self.dict['goals']) == 0:
            return
        
        command = 'INSERT INTO goals_stats (goals_stats.match, goalscorer_id, minute, goal_type, own_goalscorer_id, assistant_id) VALUES '

        #print ('\n{1} goals in match {0}'.format(self.dict['id'], len(self.dict['goals'])))
        match_id = self.dict['id']

        for [scorer, type, minute] in self.dict['goals']:
            assert type >= 0 and type <= 2, type

            if type == goal_type.own_goal:
                goalscorer_id = 'NULL'
                own_goalscorer_id = scorer
            else:
                goalscorer_id = scorer
                own_goalscorer_id = 'NULL'
    
            data = '(\'{0}\', \'{1}\',\'{2}\',\'{3}\',\'{4}\',\'{5}\'),'.format(match_id, goalscorer_id, minute, type, own_goalscorer_id, 'NULL')
            
            command = command + data
        
        command = command[:-1] + ';'
        if out_file == '':
            print (command)
        else:
            out_file.write(command)
        
    def generate_player_stats_sql(self, out_file = ''):
        
        command = 'INSERT INTO player_stats (player_stats.match, player_teamsheet_number, player_shirtnumber, player, came_on, came_off) VALUES '
        
        for data in self.dict['line-up']:
            ''' match id, player teamsheet number, shirt number, player id, came on,
            came off, captain, yellow card, second yellow, red card  '''
            values = '(\'{0}\', \'{1}\',\'{2}\',\'{3}\',\'{4}\',\'{5}\')'\
                          .format(self.dict['id'], data[0], data[1], data[2], data[3], data[4])
                          
            command = command + values + ','
        
        command = command[:-1] + ';\n'

        if out_file == '':
            print (command)
        else:
            out_file.write(command)

    

'''
Load all files in the directory
'''    

def full_load_stage():
    parser = Parse_Html_Games(remote = True)
    parser.read_files('games_html/')
    #parser.read_file(5001)


'''
LFCH

        self.stat_tags = ['Game date', 'Stadium', 'Competition', 'Attendance', 'Half-time score',\
                         'Manager', 'Opposition manager', 'Referee']
        
        self.main_tags = {  'Starting lineup':(3, 4),\
                            'Subs': (3, 4),\
                            'Goals': (1, 2, 3),\
                            'Red Cards':(2,3),\
                            'Sent off': (2,3),\
                            'Substitutions': (5, 6),\
                            'Opposition lineup': (18, 22), 
                            'Opposition substitutions': (3, 5, 6),\
                            'Opposition goals': (1, 2, 3),\
                            'Notes': range(0,10)}
                            
'''

'''
Liverweb
        self.liverpool_tags = ('Liverpool', 'Unused Subs', 'Goal(s)', 'Bookings', 'Red Cards', 'Shirt Worn', 'Manager')
        self.opposition_tags =            ( 'Unused Subs', 'Goal(s)', 'Booking',  'Red Cards', 'Shirt Worn', 'Manager')
        self.match_tags = ('Referee', 'Att', 'Venue', 'Kick Off', 'Date', 'HT/FT', 'Competition')
'''



class convert_lfch():
    def __init__(self, raw):
        self.match_data = raw
        self.liverpool = raw
    
    def all(self):
        self.dict = {}
        self.id()
        self.date()
        self.stadium()
        self.referee()
        self.attendance()
        self.tournament()
        self.half_time()
        self.score()
        self.manager()
        self.shirt()
        self.red_cards()
        return self.dict


    def parse_reference_list(self, item):
        if len(item) == 1:
            name = item[0]
            return {'name': name} 
        elif len(item) == 2:
            id = int(item[0])
            name = item[1]
            return {'id': id, 'name': name} 
        elif len(item) == 3:
            ''' minutes are present '''
            id = int(item[0])
            name = item[1]
            minute = int(item[2].strip().replace('\'', ''))
            assert minute >= 0 and minute <= 120, 'Too big minute ' + item 
            return {'id': id, 'name': name, 'minute':minute}
        else:
            print (item)
            return (item)
    
    def get_id(self, tag):
        a = self.dict[tag]
        self.dict[tag] = []
        if type(a[0]) is list:
            ''' list of lists '''
            for item in a:
                self.dict[tag].append(self.parse_reference_list(item))

        else:        
            self.dict[tag] = self.parse_reference_list(a)


            
    def id(self):
        self.dict['hid'] = int(self.match_data['hid'])
        
    def date(self):
        self.dict['Date'] = datetime.strptime(self.match_data['Game date'], '%A %d %B %Y at %H:%M')
    
    def stadium(self):
        self.dict['Stadium'] = self.match_data['Stadium']   

    def referee(self):
        if 'Referee' in self.match_data.keys():
            self.dict['Referee'] =  self.match_data['Referee']
            self.get_id('Referee') 

    def attendance(self):
        if 'Attendance' in self.match_data.keys():
            self.dict['Attendance'] =  int(self.match_data['Attendance'].replace(',','')) 
    
    def tournament(self):
        self.dict['Tournament'] = self.match_data['Competition'].rstrip()

    def half_time(self):
        if 'Half-time score' in self.match_data.keys():
            self.dict['Half-time'] = self.match_data['Half-time score']

    def score(self):
        self.match_data['Score'] = self.match_data['Score'].replace('&', 'and')
        pattern = re.compile(r'([\w\s\'-]*)\s(\d{1,2}) - (\d{1,2})\s([\w\s\'-]*)$') 
            
        try:
            parsed = pattern.search(self.match_data['Score']).groups()
        except:
            raise ValueError("Wrong Score 1")
        
        if parsed[0] == "Liverpool":
            we_score = parsed[1]
            they_score = parsed[2]
            opposition = parsed[3]
        elif parsed[3] == "Liverpool":
            opposition = parsed[0]
            they_score = parsed[1]
            we_score = parsed[2]
        else: 
            raise ValueError("Wrong Score 2")
            
        self.dict['Opposition'] = opposition
        self.dict['Scored'] = we_score
        self.dict['Conceded'] = they_score
        
    def manager(self):
        if 'Manager' in self.liverpool.keys():
            self.dict['Manager'] = self.liverpool['Manager']
            self.get_id('Manager')   

    def lineup(self):
        self.dict['Line-up'] = self.liverpool['Starting lineup']

    def shirt(self):
        pass

    def red_cards(self):
        if 'Red Cards' in self.liverpool.keys():
            source_tag = 'Red Cards'
        elif 'Sent off' in self.liverpool.keys():
            source_tag = 'Sent off'
        else:
            return
        
        self.dict['Red Cards'] = self.liverpool[source_tag]
        self.get_id('Red Cards')
        

class convert_liverweb(convert_lfch):
    def __init__(self, raw):
        self.raw = raw
        self.match_data = raw
        self.liverpool = list2dict(raw['Liverpool data'])

    def get_id(self, tag):
        self.dict[tag] = self.dict[tag].rstrip()
    
    def id(self):
        self.dict['wid'] = int(self.raw['wid'])

    def date(self):
        try:
            self.dict['Date'] =  datetime.strptime(self.match_data['Kick Off'] + ' '  + self.match_data['Date'], '%A - %H:%M %d/%m/%Y')
        except ValueError:
            ''' incorrect date format '''
            exc = {496:'30/12/1899' ,\
                   937: '07/05/1910',\
                   938: '12/05/1910',\
                   1755: '06/05/1935',\
                   1891: '09/05/1932',\
                   2432: '02/06/1948',\
                               }  
            if self.dict['wid'] in exc.keys():
                self.match_data['Date'] = exc[self.dict['wid']]
            self.dict['Date'] =  datetime.strptime(self.match_data['Kick Off'] + ' '  + self.match_data['Date'], '%A - %d/%m/%Y')
            

    def stadium(self):
        self.dict['Stadium'] = self.match_data['Venue']

    def attendance(self):
        if 'Att' in self.match_data.keys():
            self.dict['Attendance'] =  int(self.match_data['Att']) 

    def half_time(self):
        if 'HT/FT' in self.match_data.keys():
            self.dict['Half-time'] = self.match_data['HT/FT'].rstrip()

    def score(self):
        exc = {724: 'Unknown' ,\
               935: 'Unknown',
               1984: 'Unknown',\
               }
        if self.dict['wid'] in exc.keys():
            self.dict['Score'] = exc[self.dict['wid']]
            return
        
        if self.match_data['Score'][0] == 'Liverpool':
            self.dict['Opposition'] = self.match_data['Score'][2]
            self.dict['Scored']     = self.match_data['Score'][1]
            self.dict['Conceded']   = self.match_data['Score'][3]
        elif self.match_data['Score'][2] == 'Liverpool':
            self.dict['Opposition'] = self.match_data['Score'][0]
            self.dict['Scored']     = self.match_data['Score'][3]
            self.dict['Conceded']   = self.match_data['Score'][1]
        else:
            assert False, 'Wrong score ' + self.match_data['Score']

    def shirt(self):
        if 'Shirt Worn' in self.liverpool.keys():
            self.dict['Shirt'] = self.liverpool['Shirt Worn']
         

def list2dict(list):
    dict = {}
    for item in list:
        for key in item:
            dict[key] = item[key]
    return dict


def convert_stage():
    connection = pymongo.MongoClient("localhost", 27017).test
    game_l = connection.games
    game_w = connection.games_w

    for id in range(1,3):
        h_line = game_l.find_one(({'hid': '{}'.format(id)}))
        w_line = game_w.find_one(({'wid': '{}'.format(id)}))
        
        
        if h_line and w_line:
            #print (w_line)
            print ('Dict LFCH:', convert_lfch(h_line).all())
            print ('Dict LWEB:', convert_liverweb(w_line).all())
    
def full_load_stage_liverweb():
    games = Parse_LiverWeb(remote=True)
    #games.download_html('games_html_liverweb', range(1000,6800))
    games.read_files('games_html_liverweb/')
    #games.read_file('games_html_liverweb/', 851)



print ('Starting')
#full_load_stage()
#full_load_stage_liverweb()
convert_stage()

print ('\nDone')


